{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 残差网络（ResNet）\n",
    "设输入为$\\boldsymbol{x}$。假设我们希望学出的理想映射为$f(\\boldsymbol{x})$，从而作为上方激活函数的输入。左图虚线框中的部分需要直接拟合出该映射$f(\\boldsymbol{x})$，而右图虚线框中的部分则需要拟合出有关恒等映射的残差映射$f(\\boldsymbol{x})-\\boldsymbol{x}$。残差映射在实际中往往更容易优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"..\") \n",
    "\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.backends.cudnn.deterministic = False\n",
    "torch.backends.cudnn.enabled = True\n",
    "\n",
    "import dl_utils\n",
    "\n",
    "class Residual(nn.Module):  # 本类已保存在d2lzh_pytorch包中方便以后使用\n",
    "    def __init__(self, in_channels, out_channels, use_1x1conv=False, stride=1):\n",
    "        super(Residual, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, stride=stride)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)\n",
    "        if use_1x1conv:\n",
    "            self.conv3 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride)\n",
    "        else:\n",
    "            self.conv3 = None\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "\n",
    "    def forward(self, X):\n",
    "        Y = F.relu(self.bn1(self.conv1(X)))\n",
    "        Y = self.bn2(self.conv2(Y))\n",
    "        if self.conv3:\n",
    "            X = self.conv3(X)\n",
    "        return F.relu(Y + X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 3, 6, 6])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blk = Residual(3, 3)\n",
    "X = torch.rand((4, 3, 6, 6))\n",
    "blk(X).shape # torch.Size([4, 3, 6, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 6, 3, 3])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blk = Residual(3, 6, use_1x1conv=True, stride=2)\n",
    "blk(X).shape # torch.Size([4, 6, 3, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet模型\n",
    "\n",
    "ResNet的前两层跟之前介绍的GoogLeNet中的一样：在输出通道数为64、步幅为2的$7\\times 7$卷积层后接步幅为2的$3\\times 3$的最大池化层。不同之处在于ResNet每个卷积层后增加的批量归一化层。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "        nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n",
    "        nn.BatchNorm2d(64), \n",
    "        nn.ReLU(),\n",
    "        nn.MaxPool2d(kernel_size=3, stride=2, padding=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_block(in_channels, out_channels, num_residuals, first_block=False):\n",
    "    if first_block:\n",
    "        assert in_channels == out_channels # 第一个模块的通道数同输入通道数一致\n",
    "    blk = []\n",
    "    for i in range(num_residuals):\n",
    "        if i == 0 and not first_block:\n",
    "            blk.append(Residual(in_channels, out_channels, use_1x1conv=True, stride=2))\n",
    "        else:\n",
    "            blk.append(Residual(out_channels, out_channels))\n",
    "    return nn.Sequential(*blk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.add_module(\"resnet_block1\", resnet_block(64, 64, 2, first_block=True))\n",
    "net.add_module(\"resnet_block2\", resnet_block(64, 128, 2))\n",
    "net.add_module(\"resnet_block3\", resnet_block(128, 256, 2))\n",
    "net.add_module(\"resnet_block4\", resnet_block(256, 512, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 512, 7, 7])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = torch.rand((1, 1, 224, 224))\n",
    "net(X).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.add_module(\"global_avg_pool\", nn.MaxPool2d(7,7)) # GlobalAvgPool2d的输出: (Batch, 512, 1, 1)\n",
    "net.add_module(\"fc\", nn.Sequential(dl_utils.FlattenLayer(), nn.Linear(512, 10))) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里每个模块里有4个卷积层（不计算$1\\times 1$卷积层），加上最开始的卷积层和最后的全连接层，共计18层。这个模型通常也被称为ResNet-18。通过配置不同的通道数和模块里的残差块数可以得到不同的ResNet模型，例如更深的含152层的ResNet-152。虽然ResNet的主体架构跟GoogLeNet的类似，但ResNet结构更简单，修改也更方便。这些因素都导致了ResNet迅速被广泛使用。\n",
    "\n",
    "在训练ResNet之前，我们来观察一下输入形状在ResNet不同模块之间的变化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  output shape:\t torch.Size([1, 64, 112, 112])\n",
      "1  output shape:\t torch.Size([1, 64, 112, 112])\n",
      "2  output shape:\t torch.Size([1, 64, 112, 112])\n",
      "3  output shape:\t torch.Size([1, 64, 56, 56])\n",
      "resnet_block1  output shape:\t torch.Size([1, 64, 56, 56])\n",
      "resnet_block2  output shape:\t torch.Size([1, 128, 28, 28])\n",
      "resnet_block3  output shape:\t torch.Size([1, 256, 14, 14])\n",
      "resnet_block4  output shape:\t torch.Size([1, 512, 7, 7])\n",
      "global_avg_pool  output shape:\t torch.Size([1, 512, 1, 1])\n",
      "fc  output shape:\t torch.Size([1, 10])\n"
     ]
    }
   ],
   "source": [
    "X = torch.rand((1, 1, 224, 224))\n",
    "for name, layer in net.named_children():\n",
    "    X = layer(X)\n",
    "    print(name, ' output shape:\\t', X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training on  cuda\n",
      "epoch 1, loss 0.4065, train acc 0.851, test acc 0.881, time 47.0 sec\n",
      "epoch 2, loss 0.1191, train acc 0.912, test acc 0.902, time 44.4 sec\n",
      "epoch 3, loss 0.0665, train acc 0.925, test acc 0.918, time 43.6 sec\n",
      "epoch 4, loss 0.0423, train acc 0.938, test acc 0.908, time 43.3 sec\n",
      "epoch 5, loss 0.0288, train acc 0.946, test acc 0.925, time 43.9 sec\n"
     ]
    }
   ],
   "source": [
    "batch_size = 256\n",
    "# 如出现“out of memory”的报错信息，可减小batch_size或resize\n",
    "train_iter, test_iter = dl_utils.load_data_fashion_mnist(batch_size, resize=96)\n",
    "device = 'cuda'\n",
    "lr, num_epochs = 0.001, 5\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "dl_utils.train_ch5(net, train_iter, test_iter, batch_size, optimizer, device, num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 小结\n",
    "\n",
    "- 残差块通过跨层的数据通道从而能够训练出有效的深度神经网络。\n",
    "- ResNet深刻影响了后来的深度神经网络的设计。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
